package wencai;

import org.apache.commons.io.FileUtils;
import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.HttpClients;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import util.CosineSimilarityAlgorithm;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.util.*;

import static proxy.Proxy.getProxy;

/**
 * Created by zhongwei on 2017/1/5.
 */
public class Entry extends Thread {

    private static final String NO_RESULT_FOUND = "财经百科尚未收录词条";
    static FileWriter fileWriter = null;
    static Set<String> set = new HashSet<>();
    private static String QUERY_STRING = "http://www.iwencai.com/yike/search?typed=1&preParams=&ts=1&f=1&qs=index_rewrite&selfsectsn=&querytype=&searchfilter=&tid=info&w=[KEYWORD]&isdetail=1";
    private static HttpClient client = HttpClients.createDefault();
    private static String proxyHost;
    private static int proxyPort;

    public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
        fileWriter = new FileWriter("wencai_result.txt", true);
        String content = FileUtils.readFileToString(new File("wencai_result.txt"));
        String[] dones = content.split("==========\n");
        for (String str : dones) {
            set.add(str.split("\n")[0]);
        }
        List<String> peoples = FileUtils.readLines(new File("peoples"));
        int i = set.size();
        int total = peoples.size();
        for (String str : peoples) {
            if (set.contains(str))
                continue;
            System.out.println("wencai process : " + i++ + " / " + total);
            String[] arr = str.split("\t");
            if (arr.length == 2) {
                String people = arr[0];
                String url = QUERY_STRING.replace("[KEYWORD]", URLEncoder.encode(people));
                String page1 = download(url);
                if (page1.contains("系统判断您访问次数过多，已经阻止您访问")) {
                    System.err.println("被封禁了，休息一个小时再来");
                    System.exit(-1);
                }
                judge(page1, str);
            }
        }
        System.out.println("wencai all is done");
        fileWriter.close();
    }

    private static String download(String url) {
        while (true) {
            try {
                if (proxyHost == null) {
                    setProxy();
                }
                Document document = Jsoup.connect(url).proxy(proxyHost, proxyPort).timeout(5000).get();
                System.out.println("Request through proxy, host : " + proxyHost + " port : " + proxyPort);
                String content =  document.html();
                if (content.contains("系统判断您访问次数过多，已经阻止您访问")) {
                    setProxy();
                    return download(url);
                }
                return content;
            } catch (Exception e) {
                try {
                    setProxy();
                } catch (Exception e1) {

                }
            }
        }
    }
    static void setProxy() throws IOException {
        String proxy = getProxy();
        String[] arr = proxy.split(":");
        proxyHost = arr[0];
        proxyPort = Integer.parseInt(arr[1]);
    }

    private static void judge(String page1, String people) throws IOException {
        if (page1.contains(NO_RESULT_FOUND)) {
            // 没有找到词条就不用进一步处理
            fileWriter.write(people + "\n\n");
            fileWriter.write("==========\n");
            fileWriter.flush();
        } else if (page1.contains("这是一个多义词，请在下列义项中选择浏览。")) {
            // 多义词，需要根据公司名称进一步过滤
            processMultiSemantic(page1, people);
        } else {
            processDetail(page1, people);
        }
    }

    static void processMultiSemantic(String page1, String people) throws IOException {
        Document document = Jsoup.parse(page1);
        Element poly = document.getElementsByClass("polysemylist").get(0);
        Elements polyList = poly.getElementsByTag("a");
        Element mostLikelyEle = null;
        try {
            mostLikelyEle = getMostLikelyEle(polyList, people);
        } catch (Exception e) {
            System.err.println("在选择多义词的时候出现错误，忽略继续...");
        }
        if (mostLikelyEle == null) {
            fileWriter.append(people + "\n\n");
            fileWriter.write("==========\n");
            fileWriter.flush();
        } else {
            Element a = mostLikelyEle.getElementsByTag("a").get(0);
            String url = "http://www.iwencai.com" + a.attr("href");
            String detailPage = download(url);
            processDetail(detailPage, people);
        }
    }

    static Element getMostLikelyEle(Elements polyList, String people) {
        final String org = people.split("\t")[1];
        Collections.sort(polyList, new Comparator<Element>() {
            @Override
            public int compare(Element o1, Element o2) {
                double diff = CosineSimilarityAlgorithm.getSimilarity(o2.text().trim(), org) - CosineSimilarityAlgorithm.getSimilarity(o1.text(), org);
                if (diff == 0d) return 0;
                else if (diff > 0d) return 1;
                else return -1;
            }
        });
        double mostLike = CosineSimilarityAlgorithm.getSimilarity(polyList.get(0).text(), org);
        if (mostLike < 0.2d) {
            return null;
        }
        return polyList.get(0);
    }

    static void processDetail(String detailPage, String people) throws IOException {
        Document document = Jsoup.parse(detailPage);
        Elements elements = document.getElementsByClass("basicitem");
        if (elements != null) {
            fileWriter.write(people + "\n");
            for (Element ele : elements) {
                fileWriter.write(ele.text() + "\t\t");
            }
            fileWriter.write("\n==========\n");
        }
        fileWriter.flush();
    }

    @Override
    public void run() {
        try {
            main(null);
        } catch (URISyntaxException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
}
